/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package uit.ieproject.crawler;

import edu.uci.ics.crawler4j.crawler.CrawlConfig;
import edu.uci.ics.crawler4j.crawler.CrawlController;
import edu.uci.ics.crawler4j.fetcher.PageFetcher;
import edu.uci.ics.crawler4j.robotstxt.RobotstxtConfig;
import edu.uci.ics.crawler4j.robotstxt.RobotstxtServer;
import java.util.logging.Level;
import java.util.logging.Logger;
import uit.ieproject.presentation.MainFrame;

/**
 *
 * @author RedDevils
 */
public class NameControllerThread implements Runnable {

    /**
     * @param args the command line arguments
     */
    private CrawlController controller;
    private int numberOfCrawlers = 10;

    public NameControllerThread() {
    }

    @Override
    public void run() {
        String crawlStorageFolder = "/data/crawl/root";
        numberOfCrawlers = 10;

        CrawlConfig config = new CrawlConfig();

        config.setCrawlStorageFolder(crawlStorageFolder);

        config.setPolitenessDelay(1000);

        config.setMaxDepthOfCrawling(1);

        config.setMaxPagesToFetch(1000);

        config.setResumableCrawling(false);

        /*
         * Instantiate the controller for this crawl.
         */
        PageFetcher pageFetcher = new PageFetcher(config);
        RobotstxtConfig robotstxtConfig = new RobotstxtConfig();
        RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcher);
        try {
            controller = new CrawlController(config, pageFetcher, robotstxtServer);
        } catch (Exception ex) {
            Logger.getLogger(NameControllerThread.class.getName()).log(Level.SEVERE, null, ex);
        }

        controller.addSeed(MainFrame.URL);
        //NameCrawler a = new NameCrawler(URL, STR_PARAGRAPH, list);
        controller.start(NameCrawler.class, numberOfCrawlers);
        MainFrame.crawlerNameButton.setEnabled(true);
    }
}
